package com.chance.cc.crawler.development.scripts.baidu;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.filter.FilterInfo;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Html;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.jsoup.helper.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileInputStream;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.*;

public class BaiduCarSeriesCrawlerScript extends CrawlerCommonScript {
    private static Logger log = LoggerFactory.getLogger(BaiduCarSeriesCrawlerScript.class);//日志
    private static final String DOMAIN = "baidu";
    private static final String SITE = "assignTieba";
    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";

    //    https://tieba.baidu.com
    private static final String HomeRegulars = "https://tieba.baidu.com";//动态链接
    //    https://tieba.baidu.com/f?kw=%E5%AE%9D%E9%A9%ACx3&ie=utf-8&pn=0
    //    https://tieba.baidu.com/f?kw=%E5%AE%9D%E9%A9%ACx3&ie=utf-8&pn=50
    private static final String TieBaRegulars = "https://tieba.baidu.com/f\\?kw=\\S*&ie=utf-8&pn=\\d*";//贴吧首页链接
    //    https://tieba.baidu.com/p/7552887096?pn=1
    private static final String ContentRegulars = "https://tieba.baidu.com/p/\\S*\\?pn=\\d*";//帖子链接

    private static final Object obj = new Object();

    @Override
    public String domain() {
        return DOMAIN;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(HomeRegulars);
        addUrlRegular(TieBaRegulars);
        addUrlRegular(ContentRegulars);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        String site = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return SITE.equals(site);
    }

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")) {
                this.initKeyWord(crawlerRequestRecord, supportSourceRecord);
            }
        }
        return prepareLinks;
    }

    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
        Json json = internalDownloadPage.getJson();
        String msg = json.jsonPath($_type + ".msg").get();
        if (!"success".equals(msg)) {
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        List<String> keys = new ArrayList<>();
        List<String> all = json.jsonPath($_type + ".content").all();
        for (String data : all) {
            JSONObject jsonObject = JSONObject.parseObject(data);
            String keyword = jsonObject.getString("keyword");
            keys.add(keyword);
        }
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("keys", keys);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList();
        //        判断页面是否加载成功
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        if (page.getStatusCode() != 200 || (!page.isDownloadSuccess()) || page.getRawText().contains("百度安全验证")) {//|| page.getRawText().contains("方法参数错误")
            log.error("download page url == {} error status is {}", requestUrl, page.getStatusCode(), page.getStatusCode());
            if (page.getStatusCode() != 404) {//如果没有成功的下载  进行重新下载
//                    synchronized (obj) {
//                        int v = (int) new Random().nextDouble() * 1000;
//                        try {
//                            Thread.sleep(2000 + v);
//                        } catch (InterruptedException e) {
//                            e.printStackTrace();
//                        }
                        this.requestAgainCrawlerRecord(parsedLinks, crawlerRecord);
                        crawlerRecord.setNeedWashPage(false);
                        return parsedLinks;
//                    }
            }
        }

//        LinkedBlockingQueue<String> uas = send2Redis("D:\\it\\idea\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\kaola\\ua.txt");
//        crawlerRecord.tagsCreator().bizTags().addCustomKV("uas", uas);


        if (crawlerRecord.getHttpRequest().getUrl().matches(HomeRegulars)) {
            homeRequest(crawlerRecord, page, parsedLinks);//解析动态车系
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(TieBaRegulars)) {
            carSeriesRequest(crawlerRecord, page, parsedLinks);//解析车系
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars)) {
            informationRequest(crawlerRecord, page, parsedLinks);//解析文章列表
        }
        return parsedLinks;
    }


    //    解析动态车系
    private void homeRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        KVTag keys = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("keys");
        List<String> name = (List<String>) keys.getVal();
        for (int i = 0; i < name.size(); i++) {//name.size()
            String url = "https://tieba.baidu.com/f?kw=" + name.get(i) + "&ie=utf-8&pn=0";//得到车系页面
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(url)
                    .releaseTime(Long.valueOf(System.currentTimeMillis()))
//                    .httpHead("User-Agent", uas(crawlerRecord))
                    .httpHead("User-Agent",getRandomUA())
                    .httpHead("Cookie", "BIDUPSID=24F0732DC65513FA550976466C34F6E2; PSTM=1621700217; BAIDUID=8BEFF6FFCF35E729C06E9CAC424DD506:FG=1; __yjs_duid=1_e82845adc86f073b22544f1b2daa22481622447725197; wise_device=0; bdshare_firstime=1630637521812; st_key_id=17; 1306848667_FRSVideoUploadTip=1; video_bubble1306848667=1; USER_JUMP=-1; top_list=1-6190829219; BDUSS=ZxUlJaRGE0NE9UM2hJVjAzfkxWYktERXBNY29QR35hflpFV2VsUVF1c3ZsSGRoSVFBQUFBJCQAAAAAAAAAAAEAAACb7eRNMLbpwuS1xNChw~cwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC8HUGEvB1BhQV; BDUSS_BFESS=ZxUlJaRGE0NE9UM2hJVjAzfkxWYktERXBNY29QR35hflpFV2VsUVF1c3ZsSGRoSVFBQUFBJCQAAAAAAAAAAAEAAACb7eRNMLbpwuS1xNChw~cwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC8HUGEvB1BhQV; H_WISE_SIDS=107312_110085_127969_164870_176398_178384_178605_179349_179379_179454_179620_181133_181487_181588_182236_182531_183329_183625_183984_184012_184197_184321_184441_184560_184734_184793_184891_184893_185517_185650_185879_186159_186316_186563_186595_186636_186682_186743_186844_187067_187091_187195_187292_187346_187433_187449_187669_187726_187928_188031_188182_188295_188340_188426_188615_188665_188670_188714_188733_188846_188872_188992_189058_189104_189180_189232_189326_189412_189430_189679_189712_189733_189755_190114_190131_190169; STOKEN=0e60c6c2395e2a6a14150fd803de2deda04d4ea1c324346cae0c4aaa273eaabf; H_PS_PSSID=31253_26350; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; BDSFRCVID=hTAOJeC62GbRfT3HZLWjJTDZJOXGOh7TH6aohW1E-_eJ7Ar5fxKhEG0PMM8g0KAb4upTogKK0eOTHkCF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF=tRKOoCIatC_3fP36qROK5Jt3-qoJaI62aJ3M-lTvWJ5TMC_wK4R2btDnLt5mQUTn3NQZKIQe-UPbShPC-tnNLlbQDl3abCuJ-gjxBqoR3l02VM7Ie-t2yT0VQMTEaPRMW20j0l7mWnvTsxA45J7cM4IseboJLfT-0bc4KKJxbnLWeIJEjjC5D5b3Da8HJTnfa5rWst3H24nED6rlKDTjhPrM5l7rWMT-0bFHLM7X0qroObT5Q-nk3M0lyxoBhJotbHn7_JjOKtQ_HRT8K-RWLTtOD47qtUQxtN4L2CnjtpvhHCTpWR6obUPUDMo9LUvW02cdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLtK--hIIxj6u3MJtt5M_X5to05TIX3b7Efhbqsh7_bf--DlI73ftftfv0Q2n9afbgyqcqVqQxLf6xy5K_hUvjBJ3RWacf0UDbf45RVJbHQT3mXqQbbN3i-xrp3CJLWb3cWKJq8UbSMUbPBTD02-nBat-OQ6npaJ5nJq5nhMJmb67JD-50eGKftjL8tnPDVbnaK4bhKCI92tnh2-cH-UnLqMQ3X2OZ0l8KtD3hJ4jKyx7E5x_WyRORXfT4abbz3DQmWIQHDIbGyR5TXl0keMDj-Uk80mv4KKJxX-PWeIJo5t5JX4P9hUJiB5O-Ban7BhOIXKohJh7FM4tW3J0ZyxomtfQxtNRJ0DnjtnLhbC8Rjj_KjjcQeU_X5-CXaC5KW-cS5Ru_Hn7zeP7xLf4pbt-qJfbaLDcnohnuBU38OxbeWx6cWM-gMaJnBT5KaKbUolKEXb5kMpj4jtjWyPkkQN3TWxuO5bRi5R_2b4JxDn3oyT3qXp0nMPbly5jtMgOBBJ0yQ4b4OR5JjxonDh83bG7MJPKtfD7H3KC-tIKMbf5; BCLID_BFESS=7471668456551593956; BDSFRCVID_BFESS=hTAOJeC62GbRfT3HZLWjJTDZJOXGOh7TH6aohW1E-_eJ7Ar5fxKhEG0PMM8g0KAb4upTogKK0eOTHkCF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF_BFESS=tRKOoCIatC_3fP36qROK5Jt3-qoJaI62aJ3M-lTvWJ5TMC_wK4R2btDnLt5mQUTn3NQZKIQe-UPbShPC-tnNLlbQDl3abCuJ-gjxBqoR3l02VM7Ie-t2yT0VQMTEaPRMW20j0l7mWnvTsxA45J7cM4IseboJLfT-0bc4KKJxbnLWeIJEjjC5D5b3Da8HJTnfa5rWst3H24nED6rlKDTjhPrM5l7rWMT-0bFHLM7X0qroObT5Q-nk3M0lyxoBhJotbHn7_JjOKtQ_HRT8K-RWLTtOD47qtUQxtN4L2CnjtpvhHCTpWR6obUPUDMo9LUvW02cdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLtK--hIIxj6u3MJtt5M_X5to05TIX3b7Efhbqsh7_bf--DlI73ftftfv0Q2n9afbgyqcqVqQxLf6xy5K_hUvjBJ3RWacf0UDbf45RVJbHQT3mXqQbbN3i-xrp3CJLWb3cWKJq8UbSMUbPBTD02-nBat-OQ6npaJ5nJq5nhMJmb67JD-50eGKftjL8tnPDVbnaK4bhKCI92tnh2-cH-UnLqMQ3X2OZ0l8KtD3hJ4jKyx7E5x_WyRORXfT4abbz3DQmWIQHDIbGyR5TXl0keMDj-Uk80mv4KKJxX-PWeIJo5t5JX4P9hUJiB5O-Ban7BhOIXKohJh7FM4tW3J0ZyxomtfQxtNRJ0DnjtnLhbC8Rjj_KjjcQeU_X5-CXaC5KW-cS5Ru_Hn7zeP7xLf4pbt-qJfbaLDcnohnuBU38OxbeWx6cWM-gMaJnBT5KaKbUolKEXb5kMpj4jtjWyPkkQN3TWxuO5bRi5R_2b4JxDn3oyT3qXp0nMPbly5jtMgOBBJ0yQ4b4OR5JjxonDh83bG7MJPKtfD7H3KC-tIKMbf5; Hm_lvt_98b9d8c2fd6608d564bf2ac2ae642948=1635747254,1636224644,1636530867,1636621706; delPer=0; PSINO=5; BDRCVFR[-Cxg3mV_4Yc]=jzbQNU_zIP3uA-duhN8uz4WU6; BAIDUID_BFESS=8BEFF6FFCF35E729C06E9CAC424DD506:FG=1; BAIDU_WISE_UID=wapp_1636688942144_659; Hm_lpvt_98b9d8c2fd6608d564bf2ac2ae642948=1636689140; ab_sr=1.0.1_YTMyNWJjMzM1YWUyMTA3YjM1MDRmNzk0MjA0NWU3NzIzYmJhNTA5MTYwYmMzOGEwMGNmMzI1ZGM2ZTUwYjQ4YzlhZWNmNGQzMDNmMmViMWNmNGE4ZmRhOGU3MjIxY2Y0YzI2ZDhjYzg0Mjc2MjMzM2M5ZjRhN2M0NmJiZGJjMWI2ZWIwYWYwYmVmNjU1M2RhNWFiZmU2NjJjYjM0MTVkOTNlNWQ0OWUwZmZlYjRjOTM1ZTM5NGM0Y2U3NjYwZTA4; st_data=670136c62628261403c99314ce662b4adc64c7248a86112bdabdb1f177122c8d215cbc3c0b37340e41334d3d5da64b591ae815f6a9bf341c4e1f1e92d4cb05a5f96459d03bf190a309b3b4ff7f823ca55efe31191a217599f38f12a1b122fb2976b50ad712d549ee60b1cee1b6aa9c529bcb25fdae3a3a3cf2ebe763f49168e2; st_sign=686c3171")
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parsedLinks.add(record);
        }

    }

    //解析车系
    private void carSeriesRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        crawlerRecord.tagsCreator().bizTags().addCustomKV("keys", "");
        String url = page.getRequest().getUrl();
        String[] split = url.replaceAll("f\\?kw=", "").replaceAll("&ie=utf-8&pn=", "/").split("/");
            try {
            if (url.matches(TieBaRegulars)) {
                synchronized (obj) {
                    int v = (int) new Random().nextDouble() * 3000;
                    Thread.sleep(4000 + v);
                    String nextPageUrl = "https://tieba.baidu.com/f?kw=" + split[split.length - 2] + "&ie=utf-8&pn=" + (Integer.parseInt(split[split.length - 1]) + 50);
                    CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRecord)
                            .recordKey(nextPageUrl)
                            .httpUrl(nextPageUrl)
                            .releaseTime(System.currentTimeMillis())
//                            .httpHead("User-Agent", uas(crawlerRecord))
                            .httpHead("User-Agent",getRandomUA())
                            .httpHead("Cookie", "BIDUPSID=24F0732DC65513FA550976466C34F6E2; PSTM=1621700217; BAIDUID=8BEFF6FFCF35E729C06E9CAC424DD506:FG=1; __yjs_duid=1_e82845adc86f073b22544f1b2daa22481622447725197; wise_device=0; bdshare_firstime=1630637521812; st_key_id=17; 1306848667_FRSVideoUploadTip=1; video_bubble1306848667=1; USER_JUMP=-1; top_list=1-6190829219; BDUSS=ZxUlJaRGE0NE9UM2hJVjAzfkxWYktERXBNY29QR35hflpFV2VsUVF1c3ZsSGRoSVFBQUFBJCQAAAAAAAAAAAEAAACb7eRNMLbpwuS1xNChw~cwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC8HUGEvB1BhQV; BDUSS_BFESS=ZxUlJaRGE0NE9UM2hJVjAzfkxWYktERXBNY29QR35hflpFV2VsUVF1c3ZsSGRoSVFBQUFBJCQAAAAAAAAAAAEAAACb7eRNMLbpwuS1xNChw~cwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC8HUGEvB1BhQV; H_WISE_SIDS=107312_110085_127969_164870_176398_178384_178605_179349_179379_179454_179620_181133_181487_181588_182236_182531_183329_183625_183984_184012_184197_184321_184441_184560_184734_184793_184891_184893_185517_185650_185879_186159_186316_186563_186595_186636_186682_186743_186844_187067_187091_187195_187292_187346_187433_187449_187669_187726_187928_188031_188182_188295_188340_188426_188615_188665_188670_188714_188733_188846_188872_188992_189058_189104_189180_189232_189326_189412_189430_189679_189712_189733_189755_190114_190131_190169; STOKEN=0e60c6c2395e2a6a14150fd803de2deda04d4ea1c324346cae0c4aaa273eaabf; H_PS_PSSID=31253_26350; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; BDSFRCVID=hTAOJeC62GbRfT3HZLWjJTDZJOXGOh7TH6aohW1E-_eJ7Ar5fxKhEG0PMM8g0KAb4upTogKK0eOTHkCF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF=tRKOoCIatC_3fP36qROK5Jt3-qoJaI62aJ3M-lTvWJ5TMC_wK4R2btDnLt5mQUTn3NQZKIQe-UPbShPC-tnNLlbQDl3abCuJ-gjxBqoR3l02VM7Ie-t2yT0VQMTEaPRMW20j0l7mWnvTsxA45J7cM4IseboJLfT-0bc4KKJxbnLWeIJEjjC5D5b3Da8HJTnfa5rWst3H24nED6rlKDTjhPrM5l7rWMT-0bFHLM7X0qroObT5Q-nk3M0lyxoBhJotbHn7_JjOKtQ_HRT8K-RWLTtOD47qtUQxtN4L2CnjtpvhHCTpWR6obUPUDMo9LUvW02cdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLtK--hIIxj6u3MJtt5M_X5to05TIX3b7Efhbqsh7_bf--DlI73ftftfv0Q2n9afbgyqcqVqQxLf6xy5K_hUvjBJ3RWacf0UDbf45RVJbHQT3mXqQbbN3i-xrp3CJLWb3cWKJq8UbSMUbPBTD02-nBat-OQ6npaJ5nJq5nhMJmb67JD-50eGKftjL8tnPDVbnaK4bhKCI92tnh2-cH-UnLqMQ3X2OZ0l8KtD3hJ4jKyx7E5x_WyRORXfT4abbz3DQmWIQHDIbGyR5TXl0keMDj-Uk80mv4KKJxX-PWeIJo5t5JX4P9hUJiB5O-Ban7BhOIXKohJh7FM4tW3J0ZyxomtfQxtNRJ0DnjtnLhbC8Rjj_KjjcQeU_X5-CXaC5KW-cS5Ru_Hn7zeP7xLf4pbt-qJfbaLDcnohnuBU38OxbeWx6cWM-gMaJnBT5KaKbUolKEXb5kMpj4jtjWyPkkQN3TWxuO5bRi5R_2b4JxDn3oyT3qXp0nMPbly5jtMgOBBJ0yQ4b4OR5JjxonDh83bG7MJPKtfD7H3KC-tIKMbf5; BCLID_BFESS=7471668456551593956; BDSFRCVID_BFESS=hTAOJeC62GbRfT3HZLWjJTDZJOXGOh7TH6aohW1E-_eJ7Ar5fxKhEG0PMM8g0KAb4upTogKK0eOTHkCF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF_BFESS=tRKOoCIatC_3fP36qROK5Jt3-qoJaI62aJ3M-lTvWJ5TMC_wK4R2btDnLt5mQUTn3NQZKIQe-UPbShPC-tnNLlbQDl3abCuJ-gjxBqoR3l02VM7Ie-t2yT0VQMTEaPRMW20j0l7mWnvTsxA45J7cM4IseboJLfT-0bc4KKJxbnLWeIJEjjC5D5b3Da8HJTnfa5rWst3H24nED6rlKDTjhPrM5l7rWMT-0bFHLM7X0qroObT5Q-nk3M0lyxoBhJotbHn7_JjOKtQ_HRT8K-RWLTtOD47qtUQxtN4L2CnjtpvhHCTpWR6obUPUDMo9LUvW02cdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLtK--hIIxj6u3MJtt5M_X5to05TIX3b7Efhbqsh7_bf--DlI73ftftfv0Q2n9afbgyqcqVqQxLf6xy5K_hUvjBJ3RWacf0UDbf45RVJbHQT3mXqQbbN3i-xrp3CJLWb3cWKJq8UbSMUbPBTD02-nBat-OQ6npaJ5nJq5nhMJmb67JD-50eGKftjL8tnPDVbnaK4bhKCI92tnh2-cH-UnLqMQ3X2OZ0l8KtD3hJ4jKyx7E5x_WyRORXfT4abbz3DQmWIQHDIbGyR5TXl0keMDj-Uk80mv4KKJxX-PWeIJo5t5JX4P9hUJiB5O-Ban7BhOIXKohJh7FM4tW3J0ZyxomtfQxtNRJ0DnjtnLhbC8Rjj_KjjcQeU_X5-CXaC5KW-cS5Ru_Hn7zeP7xLf4pbt-qJfbaLDcnohnuBU38OxbeWx6cWM-gMaJnBT5KaKbUolKEXb5kMpj4jtjWyPkkQN3TWxuO5bRi5R_2b4JxDn3oyT3qXp0nMPbly5jtMgOBBJ0yQ4b4OR5JjxonDh83bG7MJPKtfD7H3KC-tIKMbf5; Hm_lvt_98b9d8c2fd6608d564bf2ac2ae642948=1635747254,1636224644,1636530867,1636621706; delPer=0; PSINO=5; BDRCVFR[-Cxg3mV_4Yc]=jzbQNU_zIP3uA-duhN8uz4WU6; BAIDUID_BFESS=8BEFF6FFCF35E729C06E9CAC424DD506:FG=1; BAIDU_WISE_UID=wapp_1636688942144_659; Hm_lpvt_98b9d8c2fd6608d564bf2ac2ae642948=1636689140; ab_sr=1.0.1_YTMyNWJjMzM1YWUyMTA3YjM1MDRmNzk0MjA0NWU3NzIzYmJhNTA5MTYwYmMzOGEwMGNmMzI1ZGM2ZTUwYjQ4YzlhZWNmNGQzMDNmMmViMWNmNGE4ZmRhOGU3MjIxY2Y0YzI2ZDhjYzg0Mjc2MjMzM2M5ZjRhN2M0NmJiZGJjMWI2ZWIwYWYwYmVmNjU1M2RhNWFiZmU2NjJjYjM0MTVkOTNlNWQ0OWUwZmZlYjRjOTM1ZTM5NGM0Y2U3NjYwZTA4; st_data=670136c62628261403c99314ce662b4adc64c7248a86112bdabdb1f177122c8d215cbc3c0b37340e41334d3d5da64b591ae815f6a9bf341c4e1f1e92d4cb05a5f96459d03bf190a309b3b4ff7f823ca55efe31191a217599f38f12a1b122fb2976b50ad712d549ee60b1cee1b6aa9c529bcb25fdae3a3a3cf2ebe763f49168e2; st_sign=686c3171")
                            .copyBizTags()
                            .copyResultTags()
                            .build();
                    parsedLinks.add(turnPageRequest);

                }
            }
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Series_name,split[split.length - 2]);
        try {
//        明细页解析
            String rawText = page.getRawText();
            String[] split2 = rawText.split("<ul id=\"thread_list\" class=\"threadlist_bright j_threadlist_bright\">");
            String[] split3 = null;
            if (split2.length > 1) {
                split3 = split2[1].split("<div class=\"thread_list_bottom clearfix\">");
            }
            if (split3 != null) {
                List<Selectable> nodes = new Html("<ul id=\"thread_list\" class=\"threadlist_bright j_threadlist_bright\">" + split3[0]).xpath("//li[@class=\"j_thread_list clearfix thread_item_box\"]").nodes();//所有帖子
                for (Selectable node : nodes) {
//           https://tieba.baidu.com/p/7551058517
                    String contentUrl = node.xpath(".//div[@class=\"threadlist_title pull_left j_th_tit\"]//a/@href").get();//内容链接
                    if (StringUtil.isBlank(contentUrl)) {
                        continue;
                    }
                    contentUrl = "https://tieba.baidu.com" + contentUrl + "?pn=1";
                    crawlerRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_Urls, contentUrl);
                    String pubTime = node.xpath(".//span[@class=\"threadlist_reply_date pull_right j_reply_data\"]/text()").get();//时间
                    if (StringUtil.isBlank(pubTime)) {
                        continue;
                    }
                    pubTime = pubTime.replaceAll("&#xD;", "").trim();
                    Long releaseTimeToLong = timeStrToTime(pubTime);
//                if (!isDateRange(crawlerRecord, releaseTimeToLong)) {
//                    continue;
//                }
                    KVTag filterInfoTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("comment_record_filter_info");
                    CrawlerRecord filterInfoRecord = JSON.parseObject((String) filterInfoTag.getVal(), CrawlerRecord.class);
                    CrawlerRequestRecord contentRecord = CrawlerRequestRecord.builder()//解析资讯
                            .itemPageRequest(crawlerRecord)
                            .recordKey(contentUrl)
                            .httpUrl(contentUrl)
                            .releaseTime(releaseTimeToLong)
//                            .httpHead("User-Agent", uas(crawlerRecord))
                            .httpHead("User-Agent",getRandomUA())
                            .httpHead("Cookie", "BIDUPSID=24F0732DC65513FA550976466C34F6E2; PSTM=1621700217; BAIDUID=8BEFF6FFCF35E729C06E9CAC424DD506:FG=1; __yjs_duid=1_e82845adc86f073b22544f1b2daa22481622447725197; wise_device=0; bdshare_firstime=1630637521812; st_key_id=17; 1306848667_FRSVideoUploadTip=1; video_bubble1306848667=1; USER_JUMP=-1; top_list=1-6190829219; BDUSS=ZxUlJaRGE0NE9UM2hJVjAzfkxWYktERXBNY29QR35hflpFV2VsUVF1c3ZsSGRoSVFBQUFBJCQAAAAAAAAAAAEAAACb7eRNMLbpwuS1xNChw~cwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC8HUGEvB1BhQV; BDUSS_BFESS=ZxUlJaRGE0NE9UM2hJVjAzfkxWYktERXBNY29QR35hflpFV2VsUVF1c3ZsSGRoSVFBQUFBJCQAAAAAAAAAAAEAAACb7eRNMLbpwuS1xNChw~cwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC8HUGEvB1BhQV; H_WISE_SIDS=107312_110085_127969_164870_176398_178384_178605_179349_179379_179454_179620_181133_181487_181588_182236_182531_183329_183625_183984_184012_184197_184321_184441_184560_184734_184793_184891_184893_185517_185650_185879_186159_186316_186563_186595_186636_186682_186743_186844_187067_187091_187195_187292_187346_187433_187449_187669_187726_187928_188031_188182_188295_188340_188426_188615_188665_188670_188714_188733_188846_188872_188992_189058_189104_189180_189232_189326_189412_189430_189679_189712_189733_189755_190114_190131_190169; STOKEN=0e60c6c2395e2a6a14150fd803de2deda04d4ea1c324346cae0c4aaa273eaabf; H_PS_PSSID=31253_26350; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; BDSFRCVID=hTAOJeC62GbRfT3HZLWjJTDZJOXGOh7TH6aohW1E-_eJ7Ar5fxKhEG0PMM8g0KAb4upTogKK0eOTHkCF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF=tRKOoCIatC_3fP36qROK5Jt3-qoJaI62aJ3M-lTvWJ5TMC_wK4R2btDnLt5mQUTn3NQZKIQe-UPbShPC-tnNLlbQDl3abCuJ-gjxBqoR3l02VM7Ie-t2yT0VQMTEaPRMW20j0l7mWnvTsxA45J7cM4IseboJLfT-0bc4KKJxbnLWeIJEjjC5D5b3Da8HJTnfa5rWst3H24nED6rlKDTjhPrM5l7rWMT-0bFHLM7X0qroObT5Q-nk3M0lyxoBhJotbHn7_JjOKtQ_HRT8K-RWLTtOD47qtUQxtN4L2CnjtpvhHCTpWR6obUPUDMo9LUvW02cdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLtK--hIIxj6u3MJtt5M_X5to05TIX3b7Efhbqsh7_bf--DlI73ftftfv0Q2n9afbgyqcqVqQxLf6xy5K_hUvjBJ3RWacf0UDbf45RVJbHQT3mXqQbbN3i-xrp3CJLWb3cWKJq8UbSMUbPBTD02-nBat-OQ6npaJ5nJq5nhMJmb67JD-50eGKftjL8tnPDVbnaK4bhKCI92tnh2-cH-UnLqMQ3X2OZ0l8KtD3hJ4jKyx7E5x_WyRORXfT4abbz3DQmWIQHDIbGyR5TXl0keMDj-Uk80mv4KKJxX-PWeIJo5t5JX4P9hUJiB5O-Ban7BhOIXKohJh7FM4tW3J0ZyxomtfQxtNRJ0DnjtnLhbC8Rjj_KjjcQeU_X5-CXaC5KW-cS5Ru_Hn7zeP7xLf4pbt-qJfbaLDcnohnuBU38OxbeWx6cWM-gMaJnBT5KaKbUolKEXb5kMpj4jtjWyPkkQN3TWxuO5bRi5R_2b4JxDn3oyT3qXp0nMPbly5jtMgOBBJ0yQ4b4OR5JjxonDh83bG7MJPKtfD7H3KC-tIKMbf5; BCLID_BFESS=7471668456551593956; BDSFRCVID_BFESS=hTAOJeC62GbRfT3HZLWjJTDZJOXGOh7TH6aohW1E-_eJ7Ar5fxKhEG0PMM8g0KAb4upTogKK0eOTHkCF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF_BFESS=tRKOoCIatC_3fP36qROK5Jt3-qoJaI62aJ3M-lTvWJ5TMC_wK4R2btDnLt5mQUTn3NQZKIQe-UPbShPC-tnNLlbQDl3abCuJ-gjxBqoR3l02VM7Ie-t2yT0VQMTEaPRMW20j0l7mWnvTsxA45J7cM4IseboJLfT-0bc4KKJxbnLWeIJEjjC5D5b3Da8HJTnfa5rWst3H24nED6rlKDTjhPrM5l7rWMT-0bFHLM7X0qroObT5Q-nk3M0lyxoBhJotbHn7_JjOKtQ_HRT8K-RWLTtOD47qtUQxtN4L2CnjtpvhHCTpWR6obUPUDMo9LUvW02cdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLtK--hIIxj6u3MJtt5M_X5to05TIX3b7Efhbqsh7_bf--DlI73ftftfv0Q2n9afbgyqcqVqQxLf6xy5K_hUvjBJ3RWacf0UDbf45RVJbHQT3mXqQbbN3i-xrp3CJLWb3cWKJq8UbSMUbPBTD02-nBat-OQ6npaJ5nJq5nhMJmb67JD-50eGKftjL8tnPDVbnaK4bhKCI92tnh2-cH-UnLqMQ3X2OZ0l8KtD3hJ4jKyx7E5x_WyRORXfT4abbz3DQmWIQHDIbGyR5TXl0keMDj-Uk80mv4KKJxX-PWeIJo5t5JX4P9hUJiB5O-Ban7BhOIXKohJh7FM4tW3J0ZyxomtfQxtNRJ0DnjtnLhbC8Rjj_KjjcQeU_X5-CXaC5KW-cS5Ru_Hn7zeP7xLf4pbt-qJfbaLDcnohnuBU38OxbeWx6cWM-gMaJnBT5KaKbUolKEXb5kMpj4jtjWyPkkQN3TWxuO5bRi5R_2b4JxDn3oyT3qXp0nMPbly5jtMgOBBJ0yQ4b4OR5JjxonDh83bG7MJPKtfD7H3KC-tIKMbf5; Hm_lvt_98b9d8c2fd6608d564bf2ac2ae642948=1635747254,1636224644,1636530867,1636621706; delPer=0; PSINO=5; BDRCVFR[-Cxg3mV_4Yc]=jzbQNU_zIP3uA-duhN8uz4WU6; BAIDUID_BFESS=8BEFF6FFCF35E729C06E9CAC424DD506:FG=1; BAIDU_WISE_UID=wapp_1636688942144_659; Hm_lpvt_98b9d8c2fd6608d564bf2ac2ae642948=1636689140; ab_sr=1.0.1_YTMyNWJjMzM1YWUyMTA3YjM1MDRmNzk0MjA0NWU3NzIzYmJhNTA5MTYwYmMzOGEwMGNmMzI1ZGM2ZTUwYjQ4YzlhZWNmNGQzMDNmMmViMWNmNGE4ZmRhOGU3MjIxY2Y0YzI2ZDhjYzg0Mjc2MjMzM2M5ZjRhN2M0NmJiZGJjMWI2ZWIwYWYwYmVmNjU1M2RhNWFiZmU2NjJjYjM0MTVkOTNlNWQ0OWUwZmZlYjRjOTM1ZTM5NGM0Y2U3NjYwZTA4; st_data=670136c62628261403c99314ce662b4adc64c7248a86112bdabdb1f177122c8d215cbc3c0b37340e41334d3d5da64b591ae815f6a9bf341c4e1f1e92d4cb05a5f96459d03bf190a309b3b4ff7f823ca55efe31191a217599f38f12a1b122fb2976b50ad712d549ee60b1cee1b6aa9c529bcb25fdae3a3a3cf2ebe763f49168e2; st_sign=686c3171")
                            .copyBizTags()
                            .copyResultTags()
                            .notFilterRecord()
                            .needWashed(true)
                            .build();
                    filterInfoRecord.setFilter(filterInfoRecord.getFilter());
                    filterInfoRecord.setFilterInfos(filterInfoRecord.getFilterInfos());
                    parsedLinks.add(contentRecord);
                }
            }
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
    }


    //解析文章列表
    private void informationRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        try {
            String contentUrl = page.getHtml().xpath("//a[./text()=\"下一页\"]/@href").get();
            if (contentUrl != null) {
                contentUrl = "https://tieba.baidu.com" + contentUrl;
                KVTag filterInfoTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("comment_record_filter_info");
                CrawlerRecord filterInfoRecord = JSON.parseObject((String) filterInfoTag.getVal(), CrawlerRecord.class);
                CrawlerRequestRecord contentRecord = CrawlerRequestRecord.builder()//解析资讯
                        .itemPageRequest(crawlerRecord)
                        .recordKey(contentUrl)
                        .httpUrl(contentUrl)
                        .releaseTime(System.currentTimeMillis())
//                        .httpHead("User-Agent", uas(crawlerRecord))
                        .httpHead("User-Agent",getRandomUA())
                        .httpHead("Cookie", "BIDUPSID=24F0732DC65513FA550976466C34F6E2; PSTM=1621700217; BAIDUID=8BEFF6FFCF35E729C06E9CAC424DD506:FG=1; __yjs_duid=1_e82845adc86f073b22544f1b2daa22481622447725197; wise_device=0; bdshare_firstime=1630637521812; st_key_id=17; 1306848667_FRSVideoUploadTip=1; video_bubble1306848667=1; USER_JUMP=-1; top_list=1-6190829219; BDUSS=ZxUlJaRGE0NE9UM2hJVjAzfkxWYktERXBNY29QR35hflpFV2VsUVF1c3ZsSGRoSVFBQUFBJCQAAAAAAAAAAAEAAACb7eRNMLbpwuS1xNChw~cwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC8HUGEvB1BhQV; BDUSS_BFESS=ZxUlJaRGE0NE9UM2hJVjAzfkxWYktERXBNY29QR35hflpFV2VsUVF1c3ZsSGRoSVFBQUFBJCQAAAAAAAAAAAEAAACb7eRNMLbpwuS1xNChw~cwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC8HUGEvB1BhQV; H_WISE_SIDS=107312_110085_127969_164870_176398_178384_178605_179349_179379_179454_179620_181133_181487_181588_182236_182531_183329_183625_183984_184012_184197_184321_184441_184560_184734_184793_184891_184893_185517_185650_185879_186159_186316_186563_186595_186636_186682_186743_186844_187067_187091_187195_187292_187346_187433_187449_187669_187726_187928_188031_188182_188295_188340_188426_188615_188665_188670_188714_188733_188846_188872_188992_189058_189104_189180_189232_189326_189412_189430_189679_189712_189733_189755_190114_190131_190169; STOKEN=0e60c6c2395e2a6a14150fd803de2deda04d4ea1c324346cae0c4aaa273eaabf; H_PS_PSSID=31253_26350; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; BDSFRCVID=hTAOJeC62GbRfT3HZLWjJTDZJOXGOh7TH6aohW1E-_eJ7Ar5fxKhEG0PMM8g0KAb4upTogKK0eOTHkCF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF=tRKOoCIatC_3fP36qROK5Jt3-qoJaI62aJ3M-lTvWJ5TMC_wK4R2btDnLt5mQUTn3NQZKIQe-UPbShPC-tnNLlbQDl3abCuJ-gjxBqoR3l02VM7Ie-t2yT0VQMTEaPRMW20j0l7mWnvTsxA45J7cM4IseboJLfT-0bc4KKJxbnLWeIJEjjC5D5b3Da8HJTnfa5rWst3H24nED6rlKDTjhPrM5l7rWMT-0bFHLM7X0qroObT5Q-nk3M0lyxoBhJotbHn7_JjOKtQ_HRT8K-RWLTtOD47qtUQxtN4L2CnjtpvhHCTpWR6obUPUDMo9LUvW02cdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLtK--hIIxj6u3MJtt5M_X5to05TIX3b7Efhbqsh7_bf--DlI73ftftfv0Q2n9afbgyqcqVqQxLf6xy5K_hUvjBJ3RWacf0UDbf45RVJbHQT3mXqQbbN3i-xrp3CJLWb3cWKJq8UbSMUbPBTD02-nBat-OQ6npaJ5nJq5nhMJmb67JD-50eGKftjL8tnPDVbnaK4bhKCI92tnh2-cH-UnLqMQ3X2OZ0l8KtD3hJ4jKyx7E5x_WyRORXfT4abbz3DQmWIQHDIbGyR5TXl0keMDj-Uk80mv4KKJxX-PWeIJo5t5JX4P9hUJiB5O-Ban7BhOIXKohJh7FM4tW3J0ZyxomtfQxtNRJ0DnjtnLhbC8Rjj_KjjcQeU_X5-CXaC5KW-cS5Ru_Hn7zeP7xLf4pbt-qJfbaLDcnohnuBU38OxbeWx6cWM-gMaJnBT5KaKbUolKEXb5kMpj4jtjWyPkkQN3TWxuO5bRi5R_2b4JxDn3oyT3qXp0nMPbly5jtMgOBBJ0yQ4b4OR5JjxonDh83bG7MJPKtfD7H3KC-tIKMbf5; BCLID_BFESS=7471668456551593956; BDSFRCVID_BFESS=hTAOJeC62GbRfT3HZLWjJTDZJOXGOh7TH6aohW1E-_eJ7Ar5fxKhEG0PMM8g0KAb4upTogKK0eOTHkCF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF_BFESS=tRKOoCIatC_3fP36qROK5Jt3-qoJaI62aJ3M-lTvWJ5TMC_wK4R2btDnLt5mQUTn3NQZKIQe-UPbShPC-tnNLlbQDl3abCuJ-gjxBqoR3l02VM7Ie-t2yT0VQMTEaPRMW20j0l7mWnvTsxA45J7cM4IseboJLfT-0bc4KKJxbnLWeIJEjjC5D5b3Da8HJTnfa5rWst3H24nED6rlKDTjhPrM5l7rWMT-0bFHLM7X0qroObT5Q-nk3M0lyxoBhJotbHn7_JjOKtQ_HRT8K-RWLTtOD47qtUQxtN4L2CnjtpvhHCTpWR6obUPUDMo9LUvW02cdot5yBbc8eIna5hjkbfJBQttjQn3hfIkj0DKLtK--hIIxj6u3MJtt5M_X5to05TIX3b7Efhbqsh7_bf--DlI73ftftfv0Q2n9afbgyqcqVqQxLf6xy5K_hUvjBJ3RWacf0UDbf45RVJbHQT3mXqQbbN3i-xrp3CJLWb3cWKJq8UbSMUbPBTD02-nBat-OQ6npaJ5nJq5nhMJmb67JD-50eGKftjL8tnPDVbnaK4bhKCI92tnh2-cH-UnLqMQ3X2OZ0l8KtD3hJ4jKyx7E5x_WyRORXfT4abbz3DQmWIQHDIbGyR5TXl0keMDj-Uk80mv4KKJxX-PWeIJo5t5JX4P9hUJiB5O-Ban7BhOIXKohJh7FM4tW3J0ZyxomtfQxtNRJ0DnjtnLhbC8Rjj_KjjcQeU_X5-CXaC5KW-cS5Ru_Hn7zeP7xLf4pbt-qJfbaLDcnohnuBU38OxbeWx6cWM-gMaJnBT5KaKbUolKEXb5kMpj4jtjWyPkkQN3TWxuO5bRi5R_2b4JxDn3oyT3qXp0nMPbly5jtMgOBBJ0yQ4b4OR5JjxonDh83bG7MJPKtfD7H3KC-tIKMbf5; Hm_lvt_98b9d8c2fd6608d564bf2ac2ae642948=1635747254,1636224644,1636530867,1636621706; delPer=0; PSINO=5; BDRCVFR[-Cxg3mV_4Yc]=jzbQNU_zIP3uA-duhN8uz4WU6; BAIDUID_BFESS=8BEFF6FFCF35E729C06E9CAC424DD506:FG=1; BAIDU_WISE_UID=wapp_1636688942144_659; Hm_lpvt_98b9d8c2fd6608d564bf2ac2ae642948=1636689140; ab_sr=1.0.1_YTMyNWJjMzM1YWUyMTA3YjM1MDRmNzk0MjA0NWU3NzIzYmJhNTA5MTYwYmMzOGEwMGNmMzI1ZGM2ZTUwYjQ4YzlhZWNmNGQzMDNmMmViMWNmNGE4ZmRhOGU3MjIxY2Y0YzI2ZDhjYzg0Mjc2MjMzM2M5ZjRhN2M0NmJiZGJjMWI2ZWIwYWYwYmVmNjU1M2RhNWFiZmU2NjJjYjM0MTVkOTNlNWQ0OWUwZmZlYjRjOTM1ZTM5NGM0Y2U3NjYwZTA4; st_data=670136c62628261403c99314ce662b4adc64c7248a86112bdabdb1f177122c8d215cbc3c0b37340e41334d3d5da64b591ae815f6a9bf341c4e1f1e92d4cb05a5f96459d03bf190a309b3b4ff7f823ca55efe31191a217599f38f12a1b122fb2976b50ad712d549ee60b1cee1b6aa9c529bcb25fdae3a3a3cf2ebe763f49168e2; st_sign=686c3171")
                        .copyBizTags()
                        .copyResultTags()
                        .needWashed(true)
                        .notFilterRecord()
                        .build();
                filterInfoRecord.setFilter(filterInfoRecord.getFilter());
                filterInfoRecord.setFilterInfos(filterInfoRecord.getFilterInfos());
                parsedLinks.add(contentRecord);
            }
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
    }


    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        crawlerRecord.tagsCreator().bizTags().addCustomKV("uas", null);
        if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars)) {//文章
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(interaction)) {
                crawlerDataList.addAll(articlInteractionWash(crawlerRecord, page));
            }
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {//文章链接
                crawlerDataList.addAll(articleWash(crawlerRecord, page));
            }
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(comment)) {//文章回复链接
                crawlerDataList.addAll(commentWash(crawlerRecord, page));
            }
        }

        return crawlerDataList;
    }

    //解析文章互动量
    public List<CrawlerData> articlInteractionWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        log.info("清洗文章互动量");
        String url = page.getRequest().getUrl();
        String comments = page.getHtml().xpath("//span[@class=\"red\"]/text()").get();//回复数
        String[] articleId = Pattern.compile("[^0-9]").matcher(url).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ");//url的关键字
        try {
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), interaction.enumVal(), articleId[0]))
                    .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), article.enumVal(), articleId[0]))
                    .resultLabelTag(interaction)
                    .url(url)
                    .releaseTime(System.currentTimeMillis())
                    .addContentKV(AICCommonField.Field_I_Comments, comments)
                    .flowInPipelineTag("kafka")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        return crawlerArticleDataList;
    }

    //解析文章
    public List<CrawlerData> articleWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        log.info("清洗文章");
        String url = page.getRequest().getUrl();
        String seriesName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AutoVMCommonField.Field_Series_name);
        String title = page.getHtml().xpath("//h1/text()").get();//标题
        List<Selectable> nodes = page.getHtml().xpath("//div[@class=\"l_post j_l_post l_post_bright noborder\"]").nodes();//所有回复
        for (Selectable node : nodes) {
            List<String> texts = node.xpath(".//div[@class=\"d_post_content j_d_post_content  clearfix\"]//text()").all();//文章文本
            StringBuffer conents = new StringBuffer();//将文本拼接
            for (String text : texts) {
                conents.append(text).append(" ");
            }
            List<String> picture = node.xpath(".//div[@class=\"d_post_content j_d_post_content  clearfix\"]//img/@src").all();//图片
            StringBuffer pictures = new StringBuffer();//拼接图片
            for (String text : picture) {
                pictures.append(text).append("\0x1");
            }
            String jsonStr = node.xpath("./@data-field").get().replaceAll("&quot;", "\"");
            String authorId = new Json(jsonStr).jsonPath($_type + "..user_id").get();//用户id
            String authorName = new Json(jsonStr).jsonPath($_type + "..user_name").get();//用户名
            if (authorName.equals("") || authorName == null) {
                authorName = new Json(jsonStr).jsonPath($_type + "..user_nickname").get();//评论id
            }
            String floor = new Json(jsonStr).jsonPath($_type + "..post_no").get();//楼数

            String releaseTime = new Json(jsonStr).jsonPath($_type + "..date").get();//时间
            if (StringUtils.isBlank(releaseTime)) {
                return crawlerArticleDataList;
            }
            try {
                long time = DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm").getTime();
                String dateRange = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("DateRange");
                if (dateRange != null) {
                    if (time - Long.parseLong(dateRange) > 0) {
                        return crawlerArticleDataList;
                    }
                }
                String[] articleId = Pattern.compile("[^0-9]").matcher(url).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ");//url的关键字
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), article.enumVal(), articleId[0]))
                        .resultLabelTag(article)
                        .url(url)
                        .releaseTime(time)
                        .addContentKV(AICCommonField.Field_Content, conents.toString().trim())
                        .addContentKV(AICCommonField.Field_Title, title)
                        .addContentKV(AICCommonField.Field_Author, authorName)
                        .addContentKV(AICCommonField.Field_Author_Id, authorId)
                        .addContentKV(AICCommonField.Field_Images, pictures.toString().trim())
                        .addContentKV(AICCommonField.Field_Floor, floor)
                        .addContentKV(AutoVMCommonField.Field_Series_name, seriesName)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                crawlerArticleDataList.add(crawlerData);

            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }
        return crawlerArticleDataList;
    }

    //  评论清洗
    private Collection<? extends CrawlerData> commentWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        log.info("清洗评论");
        String url = page.getRequest().getUrl();
        String seriesName = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AutoVMCommonField.Field_Series_name);
        List<Selectable> nodes = page.getHtml().xpath("//div[@class=\"l_post j_l_post l_post_bright\"]").nodes();//所有回复
        for (Selectable node : nodes) {
            List<String> texts = node.xpath(".//div[@class=\"d_post_content j_d_post_content  clearfix\"]//text()").all();//文章文本
            StringBuffer conents = new StringBuffer();//将文本拼接
            for (String text : texts) {
                conents.append(text).append(" ");
            }
            List<String> picture = node.xpath(".//div[@class=\"d_post_content j_d_post_content  clearfix\"]//img/@src").all();//图片
            StringBuffer pictures = new StringBuffer();//拼接图片
            for (String text : picture) {
                pictures.append(text).append("\0x1");
            }
            String jsonStr = node.xpath("./@data-field").get().replaceAll("&quot;", "\"");
            String authorId = new Json(jsonStr).jsonPath($_type + "..user_id").get();//用户id
            String authorName = new Json(jsonStr).jsonPath($_type + "..user_name").get();//用户名
            if (authorName.equals("") || authorName == null) {
                authorName = new Json(jsonStr).jsonPath($_type + "..user_nickname").get();//评论id
            }
            String commentId = new Json(jsonStr).jsonPath($_type + "..post_id").get();//评论id
            String floor = new Json(jsonStr).jsonPath($_type + "..post_no").get();//楼数

            String releaseTime = new Json(jsonStr).jsonPath($_type + "..date").get();//时间
            if (StringUtils.isBlank(releaseTime)) {
                return crawlerArticleDataList;
            }
            try {
                long time = DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm").getTime();
                String dateRange = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("DateRange");
                if (dateRange != null) {
                    if (time - Long.parseLong(dateRange) > 0) {
                        return crawlerArticleDataList;
                    }
                }
                String[] articleId = Pattern.compile("[^0-9]").matcher(url).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ");//url的关键字
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), comment.enumVal(), commentId))
                        .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), article.enumVal(), articleId[0]))
                        .resultLabelTag(article)
                        .url(url)
                        .releaseTime(time)
                        .addContentKV(AICCommonField.Field_Content, conents.toString().trim())
                        .addContentKV(AICCommonField.Field_Author, authorName)
                        .addContentKV(AICCommonField.Field_Author_Id, authorId)
                        .addContentKV(AICCommonField.Field_Images, pictures.toString().trim())
                        .addContentKV(AICCommonField.Field_Floor, floor)
                        .addContentKV(AutoVMCommonField.Field_Series_name, seriesName)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                crawlerArticleDataList.add(crawlerData);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }
        return crawlerArticleDataList;
    }
//
//    //  评论互动量
//    private Collection<? extends CrawlerData> commentInteractionWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
//        log.info("清洗互动量");
//        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
//        String totalStr = page.getJson().jsonPath($_type + "..total").get();
//        if (totalStr == null) {
//            totalStr = "0";
//        }
//        int total = Integer.parseInt(totalStr);
//        if (total > 0) {
//            List<Selectable> nodes = page.getJson().jsonPath($_type + "..cmntlist[*]").nodes();//所有评论
//            for (Selectable node : nodes) {
//                try {
//                    String like = new Json(node.get()).jsonPath(".rank").get();//顶
//                    String releaseTime = new Json(node.get()).jsonPath(".time").get();//回复时间
//                    long time = DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm:ss").getTime();
////                String userId = new Json(node.get()).jsonPath(".uid").get();//用户id
//                    String commentId = new Json(node.get()).jsonPath(".mid").get();//评论id
//                    String articleUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);//文章url
//
//                    CrawlerData crawlerData = CrawlerData.builder()
//                            .data(crawlerRecord, page)
//                            .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), interaction.enumVal(), commentId))
//                            .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), comment.enumVal(), commentId))
//                            .resultLabelTag(interaction)
//                            .url(articleUrl)
//                            .releaseTime(time)
//                            .addContentKV(AICCommonField.Field_I_Likes, like)
//                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
//                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
//                            .flowInPipelineTag("kafka")
//                            .build();
//                    crawlerData.setFilterPipelineResult(true);
//                    crawlerArticleDataList.add(crawlerData);
//                } catch (Exception e) {
//                    log.error(e.getMessage(), e);
//                }
//            }
//        }
//        return crawlerArticleDataList;
//    }

//    //  评论回复
//    private Collection<? extends CrawlerData> replyWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
//        log.info("清洗回复");
//        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
//        String totalStr = page.getJson().jsonPath($_type + "..total").get();
//        if (totalStr == null) {
//            totalStr = "0";
//        }
//        int total = Integer.parseInt(totalStr);
//        if (total > 0) {
//            List<Selectable> nodes = page.getJson().jsonPath($_type + "..cmntlist[*]").nodes();
//            for (Selectable node : nodes) {
//                String userId2 = new Json(node.get()).jsonPath(".parent_uid").get();//原评论用户id
//                if (userId2 != null && !"".equals(userId2)) {
//                    try {
//                        String content1 = new Json(node.get()).jsonPath(".content").get();//回复内容
//                        String releaseTime = new Json(node.get()).jsonPath(".time").get();//回复时间
//                        long time = DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm:ss").getTime();
//                        String user1 = new Json(node.get()).jsonPath(".nick").get();//回复用户
//                        String userId1 = new Json(node.get()).jsonPath(".uid").get();//回复用户id
//                        String commentId1 = new Json(node.get()).jsonPath(".mid").get();//回复评论id
//                        String commentId2 = new Json(node.get()).jsonPath(".parent").get();//原评论id
//                        String commentId = commentId1 + commentId2;
////                    String content2 = new Json(node.get()).jsonPath(".Quote.RContent").get();//原评论内容
//                        String user2 = new Json(node.get()).jsonPath(".parent_nick").get();//原评论用户
//                        String content = "回复：" + content1;
//                        String user = "回复：" + user1 + "原评论：" + user2;
//                        String userId = "回复：" + userId1 + "原评论：" + userId2;
////                        String urlId = new Json(node.get()).jsonPath(".newsid").get();//资讯url
//
//                        String url = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);//资讯内容url
//                        String[] urlId = url.replaceAll("https://k.sina.com.cn/article_", "").replaceAll(".html\\?from=auto&amp;subch=oauto", "").replaceAll(".html\\?from=auto", "").trim().split("_");//url的关键字
//                        if (url.matches(ContentTwoRegulars)) {
//                            urlId = url.replaceAll("https://t.cj.sina.com.cn/articles/view/", "").replaceAll(".html\\?from=auto&amp;subch=oauto", "").replaceAll("\\?from=auto", "").trim().split("/");
//                        }
//                        CrawlerData crawlerData = CrawlerData.builder()
//                                .data(crawlerRecord, page)
//                                .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), comment.enumVal(), commentId))
//                                .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), article.enumVal(), urlId[urlId.length - 1]))
//                                .resultLabelTag(comment)
//                                .url(url)
//                                .releaseTime(time)
//                                .addContentKV(AICCommonField.Field_Content, content)
//                                .addContentKV(AICCommonField.Field_Author, user)
//                                .addContentKV(AICCommonField.Field_Author_Id, userId)
//                                .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
//                                .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
//                                .flowInPipelineTag("kafka")
//                                .build();
//                        crawlerData.setFilterPipelineResult(true);
//                        crawlerArticleDataList.add(crawlerData);
//                    } catch (Exception e) {
//                        log.error(e.getMessage(), e);
//                    }
//                }
//            }
//        }
//        return crawlerArticleDataList;
//    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {
    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 50) {
                log.error("pcauto download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }


    public static Long timeStrToTime(String s) throws ParseException {
        Calendar now = Calendar.getInstance();
        if (s.contains(":")) {
            String releaseTime = now.get(Calendar.YEAR) + "-" + (now.get(Calendar.MONTH) + 1) + "-" + now.get(Calendar.DAY_OF_MONTH) + " " + s;
            long time = DateUtils.parseDate(releaseTime, "yyyy-MM-dd HH:mm").getTime();
            return time;
        }
        if (s.contains("-")) {
            String releaseTime = now.get(Calendar.YEAR) + "-" + s;
            long time = DateUtils.parseDate(releaseTime, "yyyy-MM-dd").getTime();
            return time;
        }
        log.error("获取时间错误");
        return null;
    }


    /**
     * 判断是否在时间范围内
     *
     * @param crawlerRequestRecord
     * @return
     */
    private boolean isDateRange(CrawlerRequestRecord crawlerRequestRecord, Long releaseTimeToLong) {
        boolean isRange = false;
        CrawlerEnum.CrawlerRecordFilter filter = crawlerRequestRecord.getFilter();
        if (filter == CrawlerEnum.CrawlerRecordFilter.keyOrDateRange || filter == CrawlerEnum.CrawlerRecordFilter.dateRange) {
            List<FilterInfo> filterInfos = crawlerRequestRecord.getFilterInfos();
            Long startTime = null;
            Long endTime = null;

            for (FilterInfo filterInfo : filterInfos) {
                if (filterInfo.getFilter() == CrawlerEnum.CrawlerRecordFilter.dateRange) {
                    long[] dateAllowRange = filterInfo.getDateAllowRange();
                    int hourFromNow = filterInfo.getHourFromNow();
                    if (dateAllowRange != null) {
                        startTime = dateAllowRange[0];
                        endTime = dateAllowRange[1];
                    } else if (hourFromNow != 0) {
                        endTime = System.currentTimeMillis() - 60000;//系统时间减去18天16小时1612800000
                        startTime = endTime - DateUtils.MILLIS_PER_HOUR * hourFromNow;
                    }
                }
            }
            if (startTime != null && releaseTimeToLong != 0 && startTime <= releaseTimeToLong && releaseTimeToLong <= endTime) {
                isRange = true;
            }
        } else {
            isRange = true;
        }
        return isRange;
    }

//    private static String uas(CrawlerRequestRecord crawlerRecord) {
//        KVTag val = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("uas");
//        LinkedBlockingQueue<String> uas = (LinkedBlockingQueue<String>) val.getVal();
//        String poll = uas.poll();
//        uas.add(poll);
//        return poll;
//    }

    //    添加ua
//    public static LinkedBlockingQueue<String> send2Redis(String filePath) {
//        List<String> uasStr = null;
//        try {
//            uasStr = IOUtils.readLines(new FileInputStream(filePath), StandardCharsets.UTF_8);
//        } catch (Exception e) {
//            System.out.println("*----------------*");
//            e.printStackTrace();
//        }
//        if (uasStr != null) {
//            LinkedBlockingQueue<String> uas = new LinkedBlockingQueue<>(200000);
//            uas.addAll(uasStr);
//            return uas;
//        } else {
//            return null;
//        }
//    }









    private static List<String> agentList = new ArrayList<>();

    static {
        agentList.add("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36");
        agentList.add("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36");
        agentList.add("Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko");
        agentList.add("Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)");
        agentList.add("Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)");
        agentList.add("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) ChromePlus/4.0.222.3 Chrome/4.0.222.3 Safari/532.2");
        agentList.add("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.28.3 (KHTML, like Gecko) Version/3.2.3 ChromePlus/4.0.222.3 Chrome/4.0.222.3 Safari/525.28.3");
        agentList.add("Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16");
        agentList.add("Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14");
        agentList.add("Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14");
        agentList.add("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14");
        agentList.add("Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02");
        agentList.add("Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00");
        agentList.add("Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00");
        agentList.add("Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00");
        agentList.add("Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1");
        agentList.add("Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0");
        agentList.add("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0");
        agentList.add("Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0");
        agentList.add("Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0");
        agentList.add("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.13 Safari/537.36");
        agentList.add("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3756.400 QQBrowser/10.5.4043.400");
    }

    private static String getRandomUA() {
        return agentList.get(RandomUtils.nextInt(0, agentList.size() - 1));
    }

}
